getEffectsAll <- function(mod) {
  pri.con.re <- ranef(mod)
  # pri.con.re$mmg1g2g3g4
  samples <- posterior_samples(mod, "mmg1g2g3g4|mmmii1mii2mii3mii4")
  prom.eff.samples <- samples[, paste0("r_", "mmg1g2g3g4", "[", rownames(promise.index), ",Intercept]")]
  
  mii.eff.samples <- samples[, paste0("r_", "mmmii1mii2mii3mii4", "[", mii.lookup$code, ",Intercept]")] 
  
  promise.index$mii <- pf$MII_cat1[match(promise.index$text, pf$Snippets)]
  promise.index$mii.code <- mii.lookup$code[match(promise.index$mii, mii.lookup$cat)]
  prom.eff.samples <- prom.eff.samples + mii.eff.samples[, promise.index$mii.code] 
  
  library(matrixStats)
  promise.summary <- dtf(promise.index, Estimate = colMeans(prom.eff.samples), 
                         se = colSds(as.matrix(prom.eff.samples)),
                         lci = colQuantiles(as.matrix(prom.eff.samples), prob = 0.025), 
                         uci = colQuantiles(as.matrix(prom.eff.samples), prob = 0.975))
  global.mean <- mean(promise.summary$Estimate)
  promise.summary$Estimate <- promise.summary$Estimate - global.mean
  promise.summary$lci <- promise.summary$lci - global.mean
  promise.summary$lci <- promise.summary$lci - global.mean
  promise.summary$est.z <- promise.summary$Estimate / sd(promise.summary$Estimate)
  promise.summary$weight <- prop.table(exp(promise.summary$est.z))
  return(promise.summary)
}

getConEffects <- function(mod) {
  pri.con.re <- ranef(mod)
  # pri.con.re$mmg1g2g3g4
  samples <- posterior_samples(mod, "mmg1g2g3g4|mmmii1mii2mii3mii4")
  prom.eff.samples <- samples[, paste0("r_", "mmg1g2g3g4", "[", rownames(promise.index), ",Intercept]")] + 
    samples[, paste0("r_", "mmg1g2g3g4", "[", rownames(promise.index), ",votecon]")]
  
  mii.eff.samples <- samples[, paste0("r_", "mmmii1mii2mii3mii4", "[", mii.lookup$code, ",Intercept]")] + 
    samples[, paste0("r_", "mmmii1mii2mii3mii4", "[", mii.lookup$code, ",votecon]")]
  
  promise.index$mii <- pf$MII_cat1[match(promise.index$text, pf$Snippets)]
  promise.index$mii.code <- mii.lookup$code[match(promise.index$mii, mii.lookup$cat)]
  prom.eff.samples <- prom.eff.samples + mii.eff.samples[, promise.index$mii.code] 
  
  library(matrixStats)
  promise.summary <- dtf(promise.index, Estimate = colMeans(prom.eff.samples), 
                         se = colSds(as.matrix(prom.eff.samples)),
                         lci = colQuantiles(as.matrix(prom.eff.samples), prob = 0.025), 
                         uci = colQuantiles(as.matrix(prom.eff.samples), prob = 0.975))
  global.mean <- mean(promise.summary$Estimate)
  promise.summary$Estimate <- promise.summary$Estimate - global.mean
  promise.summary$lci <- promise.summary$lci - global.mean
  promise.summary$lci <- promise.summary$lci - global.mean
  promise.summary$est.z <- promise.summary$Estimate / sd(promise.summary$Estimate)
  promise.summary$weight <- prop.table(exp(promise.summary$est.z))
  return(promise.summary)
}



rowToCol <- function(row) {
  
  rowGroup <- function(regex, row) {
    row.vars <- colnames( row)[grepl(regex, colnames(row) )]
    row.vars <- row.vars[!grepl("text", row.vars)]
    
    issues <- row.vars[grepl("r[[:digit:]]", row.vars)]
    outcomes <- row.vars[!grepl("r[[:digit:]]", row.vars)]
    
    
    issues <- 1:257 %in% row[, issues]
    library(mellonMisc)
    issues <- dtf(t(as.numeric(issues)))
    colnames(issues) <- paste0("issue", 1:257)
    outs <- row[, outcomes]
    colnames(outs) <- gsub(regex, "", colnames(outs))
    output <- dtf(outs, issues)
    return(output)  
  }
  
  outputs <- t(sapply(paste0("g", 1:6), row = row, rowGroup))
  outputs <- dtf(outputs)
  outputs$id <- row$ID
  outputs$page <- c(1,1,1,2,2,2)
  
  return(outputs)
}



coefSummaryFormat<- function(summary.issues) {
  coefs <- summary.issues$coefficients
  coefs <- coefs[!grepl("time", rownames(coefs)), ]
  coefs <- dtf(issue = rownames(coefs), coefs)
  coefs <- coefs[coefs$issue!="(Intercept)", ]
  coefs$issue <- gsub("issue", "", coefs$issue)
  
  
  coefs$text  <- promise.index$text[match(coefs$issue, promise.index$issue)]
  coefs <- coefs %>% arrange(Estimate)
  
  return(coefs)
}
parseFit <- function(fit) {
  estimates <- ranef(fit)[[2]][, , ]
  
  estimates <- dtf(estimates)
  estimates$text <- promise.index$text[as.numeric(rownames(estimates))]
  estimates$index <- as.numeric(rownames(estimates))
  estimates <- estimates %>% arrange(-Estimate)
  # Method 1: Add minimum value and take proportion 
  estimates$weight1 <- prop.table(estimates$Estimate - min(estimates$Estimate))
  
  # Method 2: take exponent of results
  estimates$weight2 <- prop.table(exp(estimates$Estimate))
  
  # Method 3: exponent Standardized across estimates
  estimates$weight3 <- prop.table(exp(estimates$Estimate / sd(estimates$Estimate)))
  
  # Method 4: exponent standardized by original SD
  estimates$weight4 <- prop.table(exp(estimates$Estimate / sd(promise.total$import)))
  
  return(estimates)
}

miiSumCalc <- function(imp.props, cppg=F) {
  if(!cppg) {
    mii.sums <- aggregate(t(imp.props), list(promise.index$mii), sum)
  } else {
    mii.sums <- aggregate(t(imp.props[, pf$cppg]), list(promise.index$mii[pf$cppg]), sum)  
  }
  
  issues <- mii.sums$Group.1
  mii.sums$Group.1 <- NULL
  mii.sums <- t(mii.sums)
  colnames(mii.sums) <- issues
  library(mellonMisc)
  mii.sums <- dtf(mii.sums)
  
  mii.sums <- mii.sums[, order(colMeans(mii.sums), decreasing = T)]
  return(mii.sums)
}



prepMIIPlot <- function(mii.sums, iss.num = 7, iss.exclude = NULL) {
  
  if(!is.null(iss.exclude)) {
    mii.sums <- mii.sums[, !colnames(mii.sums) %in% iss.exclude]
    colnames.bk <- colnames(mii.sums)
    mii.sums <- dtf(prop.table(as.matrix(mii.sums), 1))
    colnames(mii.sums) <- colnames.bk
  }
  
  mii.plot <- dtf(mii.sums[, 1:iss.num],
                  Other = rowSums(mii.sums[, (iss.num+1):ncol(mii.sums)]))
  library(reshape2)
  mii.plot <- melt(mii.plot)
  
  library(ggridges)
  
  colnames(mii.plot) <- c("Issue", "Proportion")
  library(scales)
  levels(mii.plot$Issue) <- Hmisc::capitalize(levels(mii.plot$Issue))
  levels(mii.plot$Issue) <- gsub("\\.", "/", levels(mii.plot$Issue))
  
  mii.plot$fill.count <- 2
  mii.plot$fill.count[(as.numeric(mii.plot$Issue)/2) %in% 1:4] <- 1
  
  mii.plot$Issue <- factor(mii.plot$Issue, levels = levels(mii.plot$Issue)[(iss.num+1):1])
  
  
  plot.medians <- tapply(mii.plot$Proportion, mii.plot$Issue, median)
  plot.medians <- dtf(median = plot.medians, issue = names(plot.medians))
  mii.plot$Median <- plot.medians$median[match(mii.plot$Issue, plot.medians$issue)]
  
  iss.unweighted <- prop.table(table(promise.index$mii[!promise.index$mii %in% iss.exclude]))
  names(iss.unweighted) <- Hmisc::capitalize(names(iss.unweighted))
  
  oth.unweighted <- sum(iss.unweighted[!names(iss.unweighted) %in% mii.plot$Issue], iss.unweighted["Other"])
  names(oth.unweighted) <- "Other"
  iss.unweighted <- iss.unweighted[names(iss.unweighted) %in% mii.plot$Issue]
  iss.unweighted <- iss.unweighted[names(iss.unweighted)!="Other"]
  iss.unweighted <- c(iss.unweighted, oth.unweighted)
  mii.plot$Unweighted <- iss.unweighted[match(mii.plot$Issue, names(iss.unweighted))]
  mii.plot$Diff <- mii.plot$Median - mii.plot$Unweighted
  
  return(mii.plot)
}


propDone <- function(imp.props, sq = "no", pf) {
  test <- imp.props
  test <- test[, pf$status_quo %in% sq]
  status.sq <- pf$statuscomb[pf$status_quo %in% sq]
  test<- prop.table(test, 1)
  test[, status.sq=="Not fulfilled"] <- 0
  return(test)
}



plotChangesFromImpProps <- function(imp.props, imp.props.c, pf) {
  done.draws <- dtf(Done_all_change = rowSums(propDone(imp.props = imp.props, "no", pf = pf)), 
                    Done_con_change = rowSums(propDone(imp.props = imp.props.c, "no", pf = pf)), 
                    Done_all_sq = rowSums(propDone(imp.props = imp.props, "yes", pf= pf)), 
                    Done_con_sq = rowSums(propDone(imp.props = imp.props.c, "yes", pf = pf)))
  
  
  weights <- colMeans(imp.props)
  pf$weight <- weights
  fulfilled <- sum(pf$weight[pf$statuscomb=="Fulfilled" & pf$status_quo=="no"], na.rm = T)
  unfulfilled <- sum(pf$weight[pf$statuscomb=="Not fulfilled" & pf$status_quo=="no"], na.rm = T)
  change.mean <- fulfilled / (fulfilled + unfulfilled)
  
  fulfilled <- sum(pf$weight[pf$statuscomb=="Fulfilled" & pf$status_quo=="yes"], na.rm = T)
  unfulfilled <- sum(pf$weight[pf$statuscomb=="Not fulfilled" & pf$status_quo=="yes"], na.rm = T)
  sq.mean <- fulfilled / (fulfilled + unfulfilled)
  
  weights.c <- colMeans(imp.props.c)
  pf$weight.c <- weights.c
  fulfilled <- sum(pf$weight.c[pf$statuscomb=="Fulfilled" & pf$status_quo=="no"], na.rm = T)
  unfulfilled <- sum(pf$weight.c[pf$statuscomb=="Not fulfilled" & pf$status_quo=="no"], na.rm = T)
  change.mean.c <- fulfilled / (fulfilled + unfulfilled)
  
  fulfilled <- sum(pf$weight.c[pf$statuscomb=="Fulfilled" & pf$status_quo=="yes"], na.rm = T)
  unfulfilled <- sum(pf$weight.c[pf$statuscomb=="Not fulfilled" & pf$status_quo=="yes"], na.rm = T)
  sq.mean.c <- fulfilled / (fulfilled + unfulfilled)
  
  
  draw.plot <- melt(done.draws)
  vars <- strsplit(as.character(draw.plot$variable), split = "_")
  draw.plot$population <- sapply(vars, function(x) x[2])
  draw.plot$type <- sapply(vars, function(x) x[3])
  draw.plot$Unweighted <- NA
  
  sqstat <- melt(prop.table(table(pf$statuscomb, pf$status_quo), 2))
  
  draw.plot$Unweighted[draw.plot$variable %in% c("Done_all_change", "Done_con_change")] <- sqstat$value[sqstat$Var1=="Fulfilled" & sqstat$Var2=="no"]
  draw.plot$Unweighted[draw.plot$variable %in% c("Done_all_sq", "Done_con_sq")] <- sqstat$value[sqstat$Var1=="Fulfilled" & sqstat$Var2=="yes"]
  
  draw.plot$Mean <- NA
  draw.plot$Mean[draw.plot$variable=="Done_all_change"] <- change.mean
  draw.plot$Mean[draw.plot$variable=="Done_con_change"] <- change.mean.c
  draw.plot$Mean[draw.plot$variable=="Done_all_sq"] <- sq.mean
  draw.plot$Mean[draw.plot$variable=="Done_con_sq"] <- sq.mean.c
  
  
  draw.plot$fill.count <- 2
  draw.plot$fill.count[(as.numeric(draw.plot$variable)/2) %in% 1:4] <- 1
  
  levels(draw.plot$variable)[levels(draw.plot$variable)=="Done_all_change"] <- "All"
  levels(draw.plot$variable)[levels(draw.plot$variable)=="Done_con_change"] <- "2017 Conservatives"
  
  draw.plot$population[draw.plot$population=="con"] <- "2017 Conservatives"
  draw.plot$population[draw.plot$population=="all"] <- "All"
  
  draw.plot$variable <- factor(draw.plot$variable, 
                               levels = levels(draw.plot$variable)[length(levels(draw.plot$variable)):1])
  draw.plot$Population <- draw.plot$population
  
  
  
  density.all.done <- ggplot(data = draw.plot[draw.plot$type=="change", ],
                             aes(x = value, group = variable, fill =Population)) + 
    scale_fill_manual(values = c("#3aa1fc", "#f49241")) +
    geom_density(colour = "white", alpha = 0.4) + 
    scale_x_continuous(labels = percent) + 
    xlab("% of change promises fulfilled\n(posterior density)") + 
    ylab("") + theme_minimal() + guides(colour = F) +
    theme(panel.grid.major.y = element_line(size=0.4, 
                                            color = "gray", linetype = 3),
          panel.grid.major.x = element_blank(),
          panel.grid.minor.x = element_blank(),
          axis.ticks.x = element_line(size=0.1, color = "black")) +
    geom_segment(aes(x=Unweighted, xend = Mean, y = Population, yend = Population),
                 arrow = arrow(length = unit(0.3, "cm")), size = 0.7, colour = "red") +
    geom_point(aes(x = Unweighted, y = Population), colour = "black", alpha = 0.1)  +
    scale_color_gradient(low = "#dd250d", high = "#18a306")
  
  draw.plot$Population <- factor(draw.plot$Population,
                                 levels = c("  ", "2017 Conservatives", "", " ", "All"))
  
  
  density.all.sq <- ggplot(data = draw.plot[draw.plot$type=="sq", ],
                           aes(x = value, group = variable, fill =Population)) + 
    scale_fill_manual(values = c("#3aa1fc", "#f49241")) +
    geom_density(colour = "white", alpha = 0.4) + 
    scale_x_continuous(labels = percent) + 
    xlab("% of status quo promises kept\n(posterior density)") + 
    ylab("") + theme_minimal() + guides(colour = F) +
    theme(panel.grid.major.y = element_line(size=0.4, 
                                            color = "gray", linetype = 3),
          panel.grid.major.x = element_blank(),
          panel.grid.minor.x = element_blank(),
          axis.ticks.x = element_line(size=0.1, color = "black")) +
    geom_segment(aes(x=Unweighted, xend = Mean, y = Population, yend = Population),
                 arrow = arrow(length = unit(0.3, "cm")), size = 0.7, colour = "red") +
    geom_point(aes(x = Unweighted, y = Population), colour = "black", alpha = 0.1)  +
    scale_color_gradient(low = "#dd250d", high = "#18a306") + 
    scale_y_discrete(drop = F, breaks = c("2017 Conservatives", "All"),
                     labels = c("2017 Conservatives", "All"))
  
  out <- list(change = density.all.done, sq = density.all.sq)
  return(out)
}

perc <- function(x) {
  percent(x, accuracy=1)
}